{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "name": "WebScrapping and PreProcessing.ipynb",
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shubhigupta991/Reddit-Flair-Detection/blob/main/scripts/WebScrapping%20and%20PreProcessing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6dtwbrrXdGcx"
      },
      "source": [
        "# Collecting the data from reddit\n",
        "\n",
        "We are collecting the data with the help of PRAW\n",
        "\n",
        "PRAW stands for Python Reddit API Wrapper."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xjMXAytddIoJ",
        "outputId": "c21b125c-d1e9-48ac-ba70-d99dc818298d"
      },
      "source": [
        "!pip install praw"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting praw\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/15/4bcc44271afce0316c73cd2ed35f951f1363a07d4d5d5440ae5eb2baad78/praw-7.1.0-py3-none-any.whl (152kB)\n",
            "\r\u001b[K     |██▏                             | 10kB 18.0MB/s eta 0:00:01\r\u001b[K     |████▎                           | 20kB 18.7MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 30kB 15.1MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 40kB 9.1MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 51kB 10.8MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 61kB 12.3MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 71kB 11.5MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 81kB 12.3MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 92kB 12.1MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 102kB 10.9MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 112kB 10.9MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 122kB 10.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 133kB 10.9MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 143kB 10.9MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 153kB 10.9MB/s \n",
            "\u001b[?25hCollecting websocket-client>=0.54.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/5f/f61b420143ed1c8dc69f9eaec5ff1ac36109d52c80de49d66e0c36c3dfdf/websocket_client-0.57.0-py2.py3-none-any.whl (200kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 19.9MB/s \n",
            "\u001b[?25hCollecting update-checker>=0.17\n",
            "  Downloading https://files.pythonhosted.org/packages/0c/ba/8dd7fa5f0b1c6a8ac62f8f57f7e794160c1f86f31c6d0fb00f582372a3e4/update_checker-0.18.0-py3-none-any.whl\n",
            "Collecting prawcore<2.0,>=1.3.0\n",
            "  Downloading https://files.pythonhosted.org/packages/1d/40/b741437ce4c7b64f928513817b29c0a615efb66ab5e5e01f66fe92d2d95b/prawcore-1.5.0-py3-none-any.whl\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from websocket-client>=0.54.0->praw) (1.15.0)\n",
            "Requirement already satisfied: requests>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from update-checker>=0.17->praw) (2.23.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.3.0->update-checker>=0.17->praw) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.3.0->update-checker>=0.17->praw) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.3.0->update-checker>=0.17->praw) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.3.0->update-checker>=0.17->praw) (2020.11.8)\n",
            "Installing collected packages: websocket-client, update-checker, prawcore, praw\n",
            "Successfully installed praw-7.1.0 prawcore-1.5.0 update-checker-0.18.0 websocket-client-0.57.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dGSFaRr4PyEN",
        "outputId": "fa179cad-ea22-4ae8-c897-6832de9fd044"
      },
      "source": [
        "import praw\n",
        "import pandas as pd\n",
        "import numpy as np \n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "import datetime as dt\n",
        "nltk.download('all')\n",
        "from bs4 import BeautifulSoup"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading collection 'all'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
            "[nltk_data]    | Downloading package alpino to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/alpino.zip.\n",
            "[nltk_data]    | Downloading package biocreative_ppi to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
            "[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n",
            "[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n",
            "[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
            "[nltk_data]    | Downloading package comparative_sentences to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n",
            "[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
            "[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/crubadan.zip.\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
            "[nltk_data]    | Downloading package dolch to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dolch.zip.\n",
            "[nltk_data]    | Downloading package europarl_raw to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n",
            "[nltk_data]    | Downloading package floresta to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/floresta.zip.\n",
            "[nltk_data]    | Downloading package framenet_v15 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n",
            "[nltk_data]    | Downloading package framenet_v17 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n",
            "[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
            "[nltk_data]    | Downloading package indian to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/indian.zip.\n",
            "[nltk_data]    | Downloading package jeita to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/kimmo.zip.\n",
            "[nltk_data]    | Downloading package knbc to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package lin_thesaurus to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n",
            "[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n",
            "[nltk_data]    | Downloading package machado to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package moses_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/moses_sample.zip.\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/names.zip.\n",
            "[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
            "[nltk_data]    | Downloading package omw to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/omw.zip.\n",
            "[nltk_data]    | Downloading package opinion_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n",
            "[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/paradigms.zip.\n",
            "[nltk_data]    | Downloading package pil to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pil.zip.\n",
            "[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pl196x.zip.\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
            "[nltk_data]    | Downloading package problem_reports to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n",
            "[nltk_data]    | Downloading package propbank to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package ptb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ptb.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_1 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n",
            "[nltk_data]    | Downloading package product_reviews_2 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n",
            "[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n",
            "[nltk_data]    | Downloading package qc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/qc.zip.\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package rte to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/rte.zip.\n",
            "[nltk_data]    | Downloading package semcor to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
            "[nltk_data]    | Downloading package sentiwordnet to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n",
            "[nltk_data]    | Downloading package sentence_polarity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n",
            "[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n",
            "[nltk_data]    | Downloading package sinica_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n",
            "[nltk_data]    | Downloading package smultron to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/smultron.zip.\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data]    | Downloading package subjectivity to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
            "[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/switchboard.zip.\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
            "[nltk_data]    | Downloading package twitter_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
            "[nltk_data]    | Downloading package universal_treebanks_v20 to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet.zip.\n",
            "[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/words.zip.\n",
            "[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ycoe.zip.\n",
            "[nltk_data]    | Downloading package rslp to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/rslp.zip.\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
            "[nltk_data]    | Downloading package sample_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n",
            "[nltk_data]    | Downloading package spanish_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n",
            "[nltk_data]    | Downloading package basque_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n",
            "[nltk_data]    | Downloading package large_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
            "[nltk_data]    | Downloading package snowball_data to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package bllip_wsj_no_aux to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n",
            "[nltk_data]    | Downloading package word2vec_sample to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping\n",
            "[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n",
            "[nltk_data]    | Downloading package perluniprops to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/perluniprops.zip.\n",
            "[nltk_data]    | Downloading package nonbreaking_prefixes to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n",
            "[nltk_data]    | Downloading package vader_lexicon to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n",
            "[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n",
            "[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection all\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jX1CxFXWQELt"
      },
      "source": [
        "reddit = praw.Reddit(client_id='KqCyLYQgMNwp4w', client_secret='cA9UCAPiVadgs4FTsnZ3RqJUR0hROw', user_agent='Flair-Detector', \n",
        "                     username='shubhigupta09', password='flair123')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AntEsya7QcRv"
      },
      "source": [
        "subreddit = reddit.subreddit('india')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cba--ygKQlF9"
      },
      "source": [
        "labels = {\"title\":[], \"id\":[], \"score\":[], \"url\":[], \"created\": [], \"num_of_comments\": [], \"body\":[], \"author\":[], \"comments\":[], \"flair\":[]}\n",
        "flairs = [\"AskIndia\", \"Non-Political\", \"[R]eddiquette\", \"Scheduled\", \"Photography\", \"Science/Technology\", \"Politics\", \n",
        "          \"Business/Finance\", \"Policy/Economy\", \"Sports\", \"Food\", \"AMA\",\"Coronavirus\"]"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3FWTHEwwRNbj"
      },
      "source": [
        "for flair in flairs:\n",
        "  \n",
        "  get_subreddits = subreddit.search(flair, limit=100)\n",
        "  \n",
        "  for each_post in get_subreddits:\n",
        "    \n",
        "    labels[\"flair\"].append(flair)\n",
        "    labels[\"title\"].append(each_post.title)\n",
        "    labels[\"score\"].append(each_post.score)\n",
        "    labels[\"id\"].append(each_post.id)\n",
        "    labels[\"url\"].append(each_post.url)\n",
        "    labels[\"num_of_comments\"].append(each_post.num_comments)\n",
        "    labels[\"created\"].append(each_post.created)\n",
        "    labels[\"body\"].append(each_post.selftext)\n",
        "    labels[\"author\"].append(each_post.author)\n",
        "    \n",
        "    each_post.comments.replace_more(limit=None)\n",
        "    comment = ''\n",
        "    for top_level_comment in each_post.comments:\n",
        "      comment = comment + ' ' + top_level_comment.body\n",
        "    labels[\"comments\"].append(comment)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m46lo67MR1QQ"
      },
      "source": [
        "def get_date(created):\n",
        "    return dt.datetime.fromtimestamp(created)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "miLXoiW3TnoJ"
      },
      "source": [
        "data = pd.DataFrame(labels)\n",
        "time =data[\"created\"].apply(get_date)\n",
        "data =data.assign(timestamp = time)\n",
        "del data['created']\n",
        "data.to_csv('reddit-india-data.csv', index=False)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 700
        },
        "id": "JIDYNrXqTrwP",
        "outputId": "a93a1db9-0bbd-42b7-e495-2001e15a722a"
      },
      "source": [
        "data=pd.read_csv('reddit-india-data.csv')\n",
        "data.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>title</th>\n",
              "      <th>id</th>\n",
              "      <th>score</th>\n",
              "      <th>url</th>\n",
              "      <th>num_of_comments</th>\n",
              "      <th>body</th>\n",
              "      <th>author</th>\n",
              "      <th>comments</th>\n",
              "      <th>flair</th>\n",
              "      <th>timestamp</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How to approach a girl?</td>\n",
              "      <td>k0qt2r</td>\n",
              "      <td>3</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/k0qt2r...</td>\n",
              "      <td>15</td>\n",
              "      <td>2 years back I was working in a startup compan...</td>\n",
              "      <td>covidmanbun</td>\n",
              "      <td>Stop watching Indian movies.  \\nStop stalking...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-11-25 19:50:12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Where is gelatine available ?</td>\n",
              "      <td>jk9zlt</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/jk9zlt...</td>\n",
              "      <td>4</td>\n",
              "      <td>I wish to buy gelatine and am looking for the ...</td>\n",
              "      <td>csstudentG</td>\n",
              "      <td>It's available in your regular kirana stores....</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-10-29 21:26:38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Trevor Noah's jokes during Indo-Pak tensions a...</td>\n",
              "      <td>gv9lmh</td>\n",
              "      <td>38</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/gv9lmh...</td>\n",
              "      <td>29</td>\n",
              "      <td>I don't really watch much of Trevor Noah's lat...</td>\n",
              "      <td>CommYouNitty</td>\n",
              "      <td>It's mostly because we as a country don't rea...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-06-02 23:42:24</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Need feedback for Insurance Policy that I took...</td>\n",
              "      <td>1s57oi</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/1s57oi...</td>\n",
              "      <td>1</td>\n",
              "      <td>**Re-posting here because of lack of activity ...</td>\n",
              "      <td>dhavalcoholic</td>\n",
              "      <td>Dear Policy Holder(Dhavalcoholic),\\n \\nWe req...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2013-12-05 14:30:23</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Buying used BS4 Scooty after April 1, Is there...</td>\n",
              "      <td>hpqq5o</td>\n",
              "      <td>12</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/hpqq5o...</td>\n",
              "      <td>7</td>\n",
              "      <td>I don't know if it's right place to ask, but \"...</td>\n",
              "      <td>akza07</td>\n",
              "      <td>Scammers will keep scamming. \\n\\nIt applies t...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-07-12 15:58:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               title  ...            timestamp\n",
              "0                            How to approach a girl?  ...  2020-11-25 19:50:12\n",
              "1                      Where is gelatine available ?  ...  2020-10-29 21:26:38\n",
              "2  Trevor Noah's jokes during Indo-Pak tensions a...  ...  2020-06-02 23:42:24\n",
              "3  Need feedback for Insurance Policy that I took...  ...  2013-12-05 14:30:23\n",
              "4  Buying used BS4 Scooty after April 1, Is there...  ...  2020-07-12 15:58:15\n",
              "\n",
              "[5 rows x 10 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POhV5u6UTvAg"
      },
      "source": [
        "def string(value):\n",
        "    return str(value)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vn4gCMiCT0j_"
      },
      "source": [
        "data['title'] = string(data['title'])\n",
        "data['body'] = string( data['body'])\n",
        "data['comments'] = string(data['comments'])"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tw1hectOT1sR"
      },
      "source": [
        "replace_by_space = re.compile('[/(){}\\[\\]\\|@,;]')\n",
        "bad_symbols = re.compile('[^0-9a-z #+_]')\n",
        "stopWords = set(stopwords.words('english'))\n",
        "def text_cleaning(text):\n",
        "   \n",
        "    text = BeautifulSoup(text, \"lxml\").text\n",
        "    text = text.lower()\n",
        "    text = replace_by_space.sub(' ', text)\n",
        "    text = bad_symbols.sub('', text)\n",
        "    text = ' '.join(word for word in text.split() if word not in stopWords)\n",
        "    return text"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dqCNZYjXT4PM"
      },
      "source": [
        "data['title'] = data['title'].apply(text_cleaning)\n",
        "data['body'] = data['body'].apply(text_cleaning)\n",
        "data['comments'] = data['comments'].apply(text_cleaning)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9avmBZKUBQG"
      },
      "source": [
        "combined_features = data[\"title\"] + data[\"comments\"] + data[\"url\"] + data[\"body\"]\n",
        "data = data.assign(combined_features = combined_features)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tuKJ0XluUDVI"
      },
      "source": [
        "data.to_csv('data.csv')"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "1GuyCluSUGKr",
        "outputId": "9f93bb24-19f5-4d10-8131-3c7594a694cc"
      },
      "source": [
        "pd.read_csv('data.csv')"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>title</th>\n",
              "      <th>id</th>\n",
              "      <th>score</th>\n",
              "      <th>url</th>\n",
              "      <th>num_of_comments</th>\n",
              "      <th>body</th>\n",
              "      <th>author</th>\n",
              "      <th>comments</th>\n",
              "      <th>flair</th>\n",
              "      <th>timestamp</th>\n",
              "      <th>combined_features</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>k0qt2r</td>\n",
              "      <td>3</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/k0qt2r...</td>\n",
              "      <td>15</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>covidmanbun</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-11-25 19:50:12</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>jk9zlt</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/jk9zlt...</td>\n",
              "      <td>4</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>csstudentG</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-10-29 21:26:38</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>gv9lmh</td>\n",
              "      <td>38</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/gv9lmh...</td>\n",
              "      <td>29</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>CommYouNitty</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-06-02 23:42:24</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>1s57oi</td>\n",
              "      <td>1</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/1s57oi...</td>\n",
              "      <td>1</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>dhavalcoholic</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2013-12-05 14:30:23</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>hpqq5o</td>\n",
              "      <td>12</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/hpqq5o...</td>\n",
              "      <td>7</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>akza07</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>AskIndia</td>\n",
              "      <td>2020-07-12 15:58:15</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1211</th>\n",
              "      <td>1211</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>j002jl</td>\n",
              "      <td>323</td>\n",
              "      <td>https://thewire.in/politics/bjp-bihar-election...</td>\n",
              "      <td>43</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>mubukugrappa</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>Coronavirus</td>\n",
              "      <td>2020-09-26 13:23:26</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1212</th>\n",
              "      <td>1212</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>j0wucw</td>\n",
              "      <td>0</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/j0wucw...</td>\n",
              "      <td>0</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>pizzapuff93</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>Coronavirus</td>\n",
              "      <td>2020-09-28 03:08:39</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1213</th>\n",
              "      <td>1213</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>izx7aw</td>\n",
              "      <td>5</td>\n",
              "      <td>https://www.reddit.com/r/india/comments/izx7aw...</td>\n",
              "      <td>1</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>spddgr8</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>Coronavirus</td>\n",
              "      <td>2020-09-26 10:12:50</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1214</th>\n",
              "      <td>1214</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>ix6pl0</td>\n",
              "      <td>1020</td>\n",
              "      <td>https://www.reddit.com/gallery/ix6pl0</td>\n",
              "      <td>153</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>IndianPuppy</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>Coronavirus</td>\n",
              "      <td>2020-09-22 02:55:01</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1215</th>\n",
              "      <td>1215</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "      <td>iwafpe</td>\n",
              "      <td>289</td>\n",
              "      <td>https://theconversation.com/india-why-secrecy-...</td>\n",
              "      <td>9</td>\n",
              "      <td>0 2 years back working startup compan1 wish bu...</td>\n",
              "      <td>9kSs</td>\n",
              "      <td>0 stop watching indian movies nstop stalking1 ...</td>\n",
              "      <td>Coronavirus</td>\n",
              "      <td>2020-09-20 16:56:43</td>\n",
              "      <td>0 approach girl1 gelatine available 2 trevor n...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1216 rows × 12 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      Unnamed: 0  ...                                  combined_features\n",
              "0              0  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "1              1  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "2              2  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "3              3  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "4              4  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "...          ...  ...                                                ...\n",
              "1211        1211  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "1212        1212  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "1213        1213  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "1214        1214  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "1215        1215  ...  0 approach girl1 gelatine available 2 trevor n...\n",
              "\n",
              "[1216 rows x 12 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9hUqQIICwtIt"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}